package crawler.web.component;

import java.util.Iterator;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Resource;

import library.database.mongo.MongoDB;
import library.thread.flow.ParameterBox;

import org.htmlparser.Attribute;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.nodes.RemarkNode;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.tags.AppletTag;
import org.htmlparser.tags.CompositeTag;
import org.htmlparser.tags.DoctypeTag;
import org.htmlparser.tags.Html;
import org.htmlparser.tags.InputTag;
import org.htmlparser.tags.JspTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.tags.StyleTag;
import org.htmlparser.util.SimpleNodeIterator;

import crawler.web.beans.CrawlPageContext;
import crawler.web.beans.CrawlURL;
import crawler.web.beans.TagData;
import crawler.web.engine.config.SpBN;
import crawler.web.engine.custom.NoiseDetector;
import crawler.web.framework.Crawler;
import crawler.web.utils.MyHttpClient;

/**
 * @author waynechen Mar 6, 2012 12:06:56 PM
 */
public class CommonHtmlProcessor implements IComponent {
	/**
	 * 通过正则表达式接受某些url
	 * 
	 * @author waynechen
	 * @param url
	 * @param pattern
	 * @return
	 */
	protected boolean accept(CrawlURL url, Pattern pattern) {
		Matcher matcher = pattern.matcher(url.getOriUrl());
		if (matcher.find())
			return true;
		return false;
	}

	/**
	 * 噪音检测器
	 */
	@Resource(name = "NoiseDetector")
	private NoiseDetector noiseDetector;

	@Resource(name = SpBN.bean_MongoDB)
	protected MongoDB mongodb;

	@Resource(name = SpBN.conf_ann_tag_table_name)
	protected String annTagTabName;

	/**
	 * 接收任何url
	 */
	@Override
	public boolean accept(CrawlURL url) {
		return true;
	}

	/**
	 * 不进行再次抓取,返回true
	 * 
	 * @param client
	 * 
	 * @param context
	 * @param parser
	 */
	@Override
	public boolean additionRequest(MyHttpClient client, CrawlPageContext context) {
		return true;
	}

	@Override
	public void process(ParameterBox params) throws Exception {
		//处理结果, 最后是否成功
		boolean ret = false;

		//处理html
		CrawlPageContext context = (CrawlPageContext) params.peek("pageContext");
		MyHttpClient client = (MyHttpClient) params.peek("client");

		//如果是爬虫的请求
		if (params.peek("caller") instanceof Crawler) {
			ret = additionRequest(client, context);
		}
		//如果是分析器的请求
		else {
			Parser parser = (Parser) params.peek("parser");
			// 下面使用一个类来执行这个过程, 并在这个类里添加多线程处理
			//遍历节点
			Html html = HtmlDigger.findHtmlNode(parser);
			if (html != null) {
				//设置文章标题
				context.getTitle().append(context.getUrl().getTitle());
				//设置url的域名
				context.setSiteDomainURL(HtmlDigger.getSiteDomainURL(context.getUrl().getOriUrl()));

				//TODO ANN标签类型判定
				//通过ann判定标签类型, 并存储入usefulNode

				//从Note到PageContext的挖掘
				dig(html, context);
			}
		}

		//将信息写入参数集中
		params.push("isSuccess", ret);
	}

	/**
	 * 通用处理<br>
	 * 
	 * @author waynechen
	 * @param root
	 * @param context
	 * @throws Exception
	 */
	protected boolean dig(Html root, CrawlPageContext context) throws Exception {
		//通用抽取有用信息
		TagData data = extract(root);
		//结构上归约
		structureReduce(data);
		//内容上归约
		contentReduce(data);
		//保存到上下文中
		context.setTagData(data);

		return true;
	}

	/**
	 * 内容归约<br>
	 * 如果某个节点没有子节点, 则将其文本追加到其父亲节点的内容后面<br>
	 * 只归约最后一层
	 * 
	 * @author waynechen
	 * @param root
	 */
	private void contentReduce(TagData root) {
		for (int i = 0; i < root.getSubTagDataList().size(); i++) {
			TagData td = root.getSubTagDataList().get(i);
			if (td.getSubTagDataList().size() > 0)
				contentReduce(td);
			else if (contentReduce(root, td))
				i--;
		}
	}

	private boolean contentReduce(TagData root, TagData td) {
		root.getTagText().append("\n" + td.getTagText());
		root.getSubTagDataList().remove(td);
		return true;
	}

	/**
	 * 如果TagData点有一个子节点, 则不要TagData, 把其子节点给TagData的父节点
	 * 
	 * @author waynechen
	 * @param root
	 */
	private void structureReduce(TagData root) {
		for (int i = 0; i < root.getSubTagDataList().size(); i++) {
			TagData td = root.getSubTagDataList().get(i);
			if (structureReduce(root, td))
				i--;
		}
	}

	private boolean structureReduce(TagData root, TagData node) {
		if (isEmptyNode(node)) {
			root.getSubTagDataList().remove(node);
		} else if (node.getSubTagDataList().size() == 1) {
			root.getSubTagDataList().remove(node);
			root.getSubTagDataList().add(node.getSubTagDataList().get(0));
			structureReduce(root, root.getSubTagDataList().get(0));
		} else if (noiseDetector.isNoise(node.getTagText().toString()) && node.getTagAttributes().length() <= 0) {
			root.getSubTagDataList().remove(node);
			root.getSubTagDataList().addAll(node.getSubTagDataList());
			for (int i = 0; i < node.getSubTagDataList().size(); i++) {
				TagData td = node.getSubTagDataList().get(i);
				if (structureReduce(root, td)) {
					node.getSubTagDataList().remove(td);
					i--;
				}
			}
		} else {
			structureReduce(node);
			return false;
		}
		return true;
	}

	private boolean isEmptyNode(TagData data) {
		if (data.getSubTagDataList().size() <= 0) {
			if (noiseDetector.isNoise(data.getTagText().toString()))
				return true;
		}
		return false;
	}

	/**
	 * 通用提取, 网页内容
	 * 
	 * @author waynechen
	 * @param root
	 * @return
	 */
	private TagData extract(Node root) {
		if (isFiltrated(root))
			return null;
		TagData tagData = new TagData();
		extractAttr(root, tagData);
		if (root instanceof CompositeTag) {
			CompositeTag ctag = (CompositeTag) root;
			SimpleNodeIterator it = ctag.elements();
			while (it.hasMoreNodes()) {
				Node next = it.nextNode();
				TagData extract = extract(next);
				if (null != extract)
					tagData.getSubTagDataList().add(extract);
			}
		} else if (root instanceof TextNode) {
			TextNode tn = (TextNode) root;
			String text = tn.getText();
			if (!noiseDetector.isJsCode(text))
				tagData.getTagText().append(HtmlDigger.collapse(text));
		}
		return tagData;
	}

	/**
	 * 获取标签上所有属性
	 * 
	 * @author waynechen
	 * @param node
	 * @param tagData
	 */
	@SuppressWarnings("unchecked")
	private void extractAttr(Node node, TagData tagData) {
		if (node instanceof TagNode) {
			TagNode tnode = (TagNode) node;
			Vector<?> vec = tnode.getAttributesEx();
			Iterator<Attribute> it = (Iterator<Attribute>) vec.iterator();
			while (it.hasNext()) {
				Attribute next = it.next();
				if (noiseDetector.isNoise(next.getName()))
					continue;
				if (noiseDetector.isNoise(next.getValue()))
					continue;
				tagData.getTagAttributes().append("(" + next.getName() + "=" + next.getValue() + ")");
			}
		}
	}

	private boolean isFiltrated(Node node) {
		if (node instanceof LinkTag)
			return true;
		else if (node instanceof ScriptTag)
			return true;
		else if (node instanceof DoctypeTag)
			return true;
		else if (node instanceof JspTag)
			return true;
		else if (node instanceof InputTag)
			return true;
		else if (node instanceof AppletTag)
			return true;
		else if (node instanceof RemarkNode)
			return true;
		else if (node instanceof StyleTag)
			return true;
		else
			return false;
	}
}

//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//